{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 包版本信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "numpy==1.13.1\n",
      "numpydoc==0.6.0\n",
      "pandas==0.20.3\n",
      "scikit-learn==0.19.0\n"
     ]
    }
   ],
   "source": [
    "!pip freeze | grep numpy\n",
    "!pip freeze | grep pandas\n",
    "!pip freeze | grep scikit-learn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>43</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>42</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   X1  X2  X3  X4  X5  X6  X7  X8  label\n",
       "0  24   2   3   3   1   1   3   0      1\n",
       "1  45   1   3  10   1   1   4   0      1\n",
       "2  43   2   3   7   1   1   4   0      1\n",
       "3  42   3   2   9   1   1   3   0      1\n",
       "4  36   3   3   8   1   1   2   0      1"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('contraception.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1473, 9)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "X1       0\n",
       "X2       0\n",
       "X3       0\n",
       "X4       0\n",
       "X5       0\n",
       "X6       0\n",
       "X7       0\n",
       "X8       0\n",
       "label    0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 检查有无缺失值\n",
    "np.sum(data.isnull())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    629\n",
       "3    511\n",
       "2    333\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 标签label的取值1，2，3\n",
    "data['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 3])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_label = np.unique(data['label'])\n",
    "class_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1473 entries, 0 to 1472\n",
      "Data columns (total 9 columns):\n",
      "X1    1473 non-null int64\n",
      "X2    1473 non-null int64\n",
      "X3    1473 non-null int64\n",
      "X4    1473 non-null int64\n",
      "X5    1473 non-null int64\n",
      "X6    1473 non-null int64\n",
      "X7    1473 non-null int64\n",
      "X8    1473 non-null int64\n",
      "Y     1473 non-null int64\n",
      "dtypes: int64(9)\n",
      "memory usage: 103.6 KB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1473.000000</td>\n",
       "      <td>1473.000000</td>\n",
       "      <td>1473.000000</td>\n",
       "      <td>1473.000000</td>\n",
       "      <td>1473.000000</td>\n",
       "      <td>1473.000000</td>\n",
       "      <td>1473.000000</td>\n",
       "      <td>1473.000000</td>\n",
       "      <td>1473.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>32.538357</td>\n",
       "      <td>2.958588</td>\n",
       "      <td>3.429735</td>\n",
       "      <td>3.261371</td>\n",
       "      <td>0.850645</td>\n",
       "      <td>0.749491</td>\n",
       "      <td>3.133741</td>\n",
       "      <td>0.073999</td>\n",
       "      <td>1.919891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>8.227245</td>\n",
       "      <td>1.014994</td>\n",
       "      <td>0.816349</td>\n",
       "      <td>2.358549</td>\n",
       "      <td>0.356559</td>\n",
       "      <td>0.433453</td>\n",
       "      <td>0.976161</td>\n",
       "      <td>0.261858</td>\n",
       "      <td>0.876376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>16.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>26.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>32.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>39.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>49.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                X1           X2           X3           X4           X5  \\\n",
       "count  1473.000000  1473.000000  1473.000000  1473.000000  1473.000000   \n",
       "mean     32.538357     2.958588     3.429735     3.261371     0.850645   \n",
       "std       8.227245     1.014994     0.816349     2.358549     0.356559   \n",
       "min      16.000000     1.000000     1.000000     0.000000     0.000000   \n",
       "25%      26.000000     2.000000     3.000000     1.000000     1.000000   \n",
       "50%      32.000000     3.000000     4.000000     3.000000     1.000000   \n",
       "75%      39.000000     4.000000     4.000000     4.000000     1.000000   \n",
       "max      49.000000     4.000000     4.000000    16.000000     1.000000   \n",
       "\n",
       "                X6           X7           X8        label  \n",
       "count  1473.000000  1473.000000  1473.000000  1473.000000  \n",
       "mean      0.749491     3.133741     0.073999     1.919891  \n",
       "std       0.433453     0.976161     0.261858     0.876376  \n",
       "min       0.000000     1.000000     0.000000     1.000000  \n",
       "25%       0.000000     3.000000     0.000000     1.000000  \n",
       "50%       1.000000     3.000000     0.000000     2.000000  \n",
       "75%       1.000000     4.000000     0.000000     3.000000  \n",
       "max       1.000000     4.000000     1.000000     3.000000  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 数据离散化\n",
    "\n",
    "对于年龄特征，我们首先将其划分为4个子区间，然后进行数字编码。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X1__(15.999, 26.0]</th>\n",
       "      <th>X1__(26.0, 32.0]</th>\n",
       "      <th>X1__(32.0, 39.0]</th>\n",
       "      <th>X1__(39.0, 49.0]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   X1__(15.999, 26.0]  X1__(26.0, 32.0]  X1__(32.0, 39.0]  X1__(39.0, 49.0]\n",
       "0                   1                 0                 0                 0\n",
       "1                   0                 0                 0                 1\n",
       "2                   0                 0                 0                 1\n",
       "3                   0                 0                 0                 1\n",
       "4                   0                 0                 1                 0"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "discrete_x1= pd.get_dummies(pd.qcut(data['X1'], 4), prefix='X1_')\n",
    "discrete_x1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X1__(15.999, 26.0]</th>\n",
       "      <th>X1__(26.0, 32.0]</th>\n",
       "      <th>X1__(32.0, 39.0]</th>\n",
       "      <th>X1__(39.0, 49.0]</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   X1__(15.999, 26.0]  X1__(26.0, 32.0]  X1__(32.0, 39.0]  X1__(39.0, 49.0]  \\\n",
       "0                   1                 0                 0                 0   \n",
       "1                   0                 0                 0                 1   \n",
       "2                   0                 0                 0                 1   \n",
       "3                   0                 0                 0                 1   \n",
       "4                   0                 0                 1                 0   \n",
       "\n",
       "   X2  X3  X4  X5  X6  X7  X8  label  \n",
       "0   2   3   3   1   1   3   0      1  \n",
       "1   1   3  10   1   1   4   0      1  \n",
       "2   2   3   7   1   1   4   0      1  \n",
       "3   3   2   9   1   1   3   0      1  \n",
       "4   3   3   8   1   1   2   0      1  "
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_data = pd.concat([discrete_x1.reset_index(drop=True), \n",
    "                        data.iloc[:,1:].reset_index(drop=True)], axis = 1)\n",
    "                      \n",
    "new_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 分类建模\n",
    "\n",
    "\n",
    "### 逻辑回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/LAMB/anaconda/lib/python2.7/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model.logistic import LogisticRegression\n",
    "from sklearn.metrics import classification_report, accuracy_score, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(pred,test_y):\n",
    "    \n",
    "    import seaborn as sns\n",
    "    import matplotlib.pyplot as plt\n",
    "    import sklearn.metrics as metrics\n",
    "    # 输出分类的准确率\n",
    "    print(\"Accuracy: %.4f\"  % (metrics.accuracy_score(test_y,pred)))\n",
    "    \n",
    "    # 输出衡量分类效果的各项指标\n",
    "    print(classification_report(test_y, pred)) \n",
    "    \n",
    "    # 更直观的，我们通过seaborn画出混淆矩阵\n",
    "    %matplotlib inline\n",
    "    plt.figure(figsize=(6,4))\n",
    "    colorMetrics = metrics.confusion_matrix(test_y,pred)\n",
    "    \n",
    "    # 坐标y代表test_y，即真实的类别，坐标x代表估计出的类别pred\n",
    "    sns.heatmap(colorMetrics, annot=True, fmt='d', xticklabels=[1,2,3],yticklabels=[1,2,3])\n",
    "    sns.plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## 数据准备\n",
    "\n",
    "X = new_data.iloc[:, :-1]\n",
    "y = new_data['label']\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'C': [0.001, 0.01, 0.1, 1, 10]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = LogisticRegression()\n",
    "param_dict = {'C': [0.001, 0.01, 0.1, 1, 10]}\n",
    "gs = GridSearchCV(lr, param_grid = param_dict, scoring='accuracy')\n",
    "gs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>mean_train_score</th>\n",
       "      <th>param_C</th>\n",
       "      <th>params</th>\n",
       "      <th>rank_test_score</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split0_train_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split1_train_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split2_train_score</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>std_train_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.004138</td>\n",
       "      <td>0.000615</td>\n",
       "      <td>0.413949</td>\n",
       "      <td>0.415773</td>\n",
       "      <td>0.001</td>\n",
       "      <td>{u'C': 0.001}</td>\n",
       "      <td>5</td>\n",
       "      <td>0.403794</td>\n",
       "      <td>0.425850</td>\n",
       "      <td>0.426630</td>\n",
       "      <td>0.422554</td>\n",
       "      <td>0.411444</td>\n",
       "      <td>0.398915</td>\n",
       "      <td>0.002324</td>\n",
       "      <td>0.000246</td>\n",
       "      <td>0.009495</td>\n",
       "      <td>0.011996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.003097</td>\n",
       "      <td>0.000450</td>\n",
       "      <td>0.456522</td>\n",
       "      <td>0.461053</td>\n",
       "      <td>0.01</td>\n",
       "      <td>{u'C': 0.01}</td>\n",
       "      <td>4</td>\n",
       "      <td>0.436314</td>\n",
       "      <td>0.473469</td>\n",
       "      <td>0.480978</td>\n",
       "      <td>0.440217</td>\n",
       "      <td>0.452316</td>\n",
       "      <td>0.469471</td>\n",
       "      <td>0.000137</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>0.018486</td>\n",
       "      <td>0.014823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.004288</td>\n",
       "      <td>0.000531</td>\n",
       "      <td>0.523551</td>\n",
       "      <td>0.537594</td>\n",
       "      <td>0.1</td>\n",
       "      <td>{u'C': 0.1}</td>\n",
       "      <td>3</td>\n",
       "      <td>0.520325</td>\n",
       "      <td>0.544218</td>\n",
       "      <td>0.529891</td>\n",
       "      <td>0.532609</td>\n",
       "      <td>0.520436</td>\n",
       "      <td>0.535957</td>\n",
       "      <td>0.000097</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>0.004484</td>\n",
       "      <td>0.004879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.006413</td>\n",
       "      <td>0.000854</td>\n",
       "      <td>0.525362</td>\n",
       "      <td>0.544833</td>\n",
       "      <td>1</td>\n",
       "      <td>{u'C': 1}</td>\n",
       "      <td>2</td>\n",
       "      <td>0.525745</td>\n",
       "      <td>0.541497</td>\n",
       "      <td>0.524457</td>\n",
       "      <td>0.543478</td>\n",
       "      <td>0.525886</td>\n",
       "      <td>0.549525</td>\n",
       "      <td>0.000391</td>\n",
       "      <td>0.000233</td>\n",
       "      <td>0.000643</td>\n",
       "      <td>0.003415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.005807</td>\n",
       "      <td>0.001747</td>\n",
       "      <td>0.528080</td>\n",
       "      <td>0.542119</td>\n",
       "      <td>10</td>\n",
       "      <td>{u'C': 10}</td>\n",
       "      <td>1</td>\n",
       "      <td>0.525745</td>\n",
       "      <td>0.537415</td>\n",
       "      <td>0.529891</td>\n",
       "      <td>0.550272</td>\n",
       "      <td>0.528610</td>\n",
       "      <td>0.538670</td>\n",
       "      <td>0.000629</td>\n",
       "      <td>0.001448</td>\n",
       "      <td>0.001735</td>\n",
       "      <td>0.005788</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  mean_score_time  mean_test_score  mean_train_score param_C  \\\n",
       "0       0.004138         0.000615         0.413949          0.415773   0.001   \n",
       "1       0.003097         0.000450         0.456522          0.461053    0.01   \n",
       "2       0.004288         0.000531         0.523551          0.537594     0.1   \n",
       "3       0.006413         0.000854         0.525362          0.544833       1   \n",
       "4       0.005807         0.001747         0.528080          0.542119      10   \n",
       "\n",
       "          params  rank_test_score  split0_test_score  split0_train_score  \\\n",
       "0  {u'C': 0.001}                5           0.403794            0.425850   \n",
       "1   {u'C': 0.01}                4           0.436314            0.473469   \n",
       "2    {u'C': 0.1}                3           0.520325            0.544218   \n",
       "3      {u'C': 1}                2           0.525745            0.541497   \n",
       "4     {u'C': 10}                1           0.525745            0.537415   \n",
       "\n",
       "   split1_test_score  split1_train_score  split2_test_score  \\\n",
       "0           0.426630            0.422554           0.411444   \n",
       "1           0.480978            0.440217           0.452316   \n",
       "2           0.529891            0.532609           0.520436   \n",
       "3           0.524457            0.543478           0.525886   \n",
       "4           0.529891            0.550272           0.528610   \n",
       "\n",
       "   split2_train_score  std_fit_time  std_score_time  std_test_score  \\\n",
       "0            0.398915      0.002324        0.000246        0.009495   \n",
       "1            0.469471      0.000137        0.000032        0.018486   \n",
       "2            0.535957      0.000097        0.000083        0.004484   \n",
       "3            0.549525      0.000391        0.000233        0.000643   \n",
       "4            0.538670      0.000629        0.001448        0.001735   \n",
       "\n",
       "   std_train_score  \n",
       "0         0.011996  \n",
       "1         0.014823  \n",
       "2         0.004879  \n",
       "3         0.003415  \n",
       "4         0.005788  "
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(gs.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 10}\n",
      "0.528079710145\n"
     ]
    }
   ],
   "source": [
    "print gs.best_params_\n",
    "print gs.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.531663097626\n"
     ]
    }
   ],
   "source": [
    "lr_cls = gs.best_estimator_\n",
    "score = cross_val_score(lr_cls, X_train, y_train, cv = 10, scoring = 'accuracy')\n",
    "print np.mean(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.4851\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          1       0.63      0.53      0.57       194\n",
      "          2       0.26      0.40      0.31        48\n",
      "          3       0.43      0.46      0.44       127\n",
      "\n",
      "avg / total       0.52      0.49      0.50       369\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAD3CAYAAABYUUzPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFbJJREFUeJzt3XmYVNW1xuFfVXU3DS2DAjIHhytLcAAFBQUFBIwTKt7c\nxDiSq6AEc4MaCSpEoyEamRIRJaACakxAgxOKKIOAUUAgBETYAQc0RFBk7mbq7rp/dEnQaFcT6vQ5\ntft7n6ceq05Z26Xgx+p19jkVSyaTiIhI5sXDLkBExFcKWBGRgChgRUQCooAVEQmIAlZEJCA5QS5+\ncvPO2qIQsJlP3xt2Cd6bPWlJ2CVUCZePuzl2qGscTOYsXzf3kP956aiDFREJSKAdrIhIZYrFAm9K\nD4oCVkS8EYtF64dyBayIeCOOOlgRkUBoRCAiEpC4RgQiIsGIWgcbrbgXEfGIOlgR8UYilgi7hK9Q\nwIqIN6I2IlDAiog34hELWM1gRUQCog5WRLwRi1jPqIAVEW8k4gpYEZFAxCJ2qWy04l5ExCPqYEXE\nG7pUVkQkINoHKyISkKjtg1XAiog3onaSSwErIt7QDFZEJCCawYqIBEQzWBGRgGgGKyISEI0IREQC\nohGBiEhAMjUiMLNqwATgGGA70B9IAhNTf30X6O+cKy1vnWjtaRAROQTxWLzCjzT6ADudcx2AnwAP\nASOBwc65s4AYcEnaeg7x30dExEetgOkAzjkHtATaAnNT708HuqdbRCMCEfFGBk9yLQMuMrPngfZA\nE+Az51wy9f4OoHa6RRSwIuKNROau5Hqcsq51PvAXYAnQ+ID3awJb0y2iEYGIeCMei1X4kcZpwCzn\nXCfgGeAD4K9m1iX1/vmUhW+5qmwHe1KblgwYdAPXXT6AZs2bcO+IQZCEte5Dhg4ZRTKZ5Krr/ofz\ne54DwPw5Cxj7u0khV519iouLuW/CJDZs2sS+4mKuuehCGtQ9guFPPEUinqBZwwb8vPc1xCP2VR/Z\npuV5p9Gk9THEcxKsfeNvfPCXlQA0P904rmsbZv5mcsgVZp01wL1mdidlnep1wGHAeDPLA1YBz6Zb\npEoG7I9u+CEXXXYuu4p2AXDbkP48NPwxFi9YxuCht9D13E6499Zy4aXdufKSfpSWljLpzw8xa8Z8\n1qz+IOTqs8uMtxdSu6CAIX2uY/vOQn509z1Y8+/Q++KenHHySdwzbjxvL19Bxzatwy41ax3Zoin1\njm3MzAcmk5OXy/HntgWgTrP6HNPxRIjY3tAgZWoG65zbxDefxOp8MOtUybbhk4/Xc/MNg/e/bnlS\nCxYvWAbAm28spEOntmz89DP6XTOQ0tKybW45OTns3bM3lHqzWdfT2nJ9r0sBSJIkEY9zXPPvsL2w\nkGQySdHuPeQkEiFXmd0antCcbes30anfxZx10yWsX/4heQX5nNyrI0unvBF2eZUqgyOCjKiSHezM\n6fNo3LTh/tcH/qlXVFjEYTULKC4uYeuWbQDcemc/Vq9cw7oP/1HptWa7Gvn5ABTt2s2Qh8fS57JL\niQEj//A0T7z0MgXVq9PmeAu3yCxX7bDqFNStxbzRz1NQrzZn33QJ2z7dzLIp8yjZVxx2eZVK9yKI\noGTpvy7GqFFQgx3bdwKQVy2Pe4b9nMKdRQwdPCqs8rLexs2buXP0w/Q6pws9OrSn509vYcyggRzd\npAlTZ81hzJ+mcMvVV4ZdZtbas3M32zdsprSklB0bt1D98JqUlpbS7spziOfmULvREZzy/c78dcrc\n9Itluay6VNbM5gDVvnY4BiSdc2cGVlUlW71yLe06tGHxgmV06tKed97+KwC/Gz+URW8tZcLYP4Zc\nYfbavG07t44YxYArr6Bdq5YA1CoooEb16gDUq1ObFWvXhlli1tu0dj0tup2Ce30p+bUL2LV1J6/e\n/STJZJKCurU4o88FVSJcIftu9jIIGA/0Arz9WWP4r8Zw1/23kZuXywdr1/H6K3M557tn0a59a/Ly\ncunUpT0Av3tgPMuXrgy52uzy5MuvsKOwiEkvTWPSS9MAGNj7au4eO45EPEFuToKBva8Jucrs9s8V\nH1K/RVN63PFDYrEYS56eTTKZTP9BD0Wtg42l+4Uws9uAtc655w528ZObd66av8qVaObT94Zdgvdm\nT1oSdglVwuXjbj7kdLzuzP4VzpzH3hoTeBqnncE654YFXYSISCZErYPVSS4R8Ua2zWBFRLKGOlgR\nkYCogxURCUjULjSokpfKiohUBnWwIuKNeLQaWAWsiPgjEbHbXipgRcQbUTvJFa24FxHxiDpYEfFG\nPGK7CBSwIuKNqI0IFLAi4g1dySUiEpCI5asCVkT8oQ5WRCQgUbtUVgErIt7QSS4RkYBoRCAiEpCI\n5asCVkT8oQ5WRCQgOsklIhIQdbAiIgHJVL6aWW+gd+plPtAGOAOYBqxJHX/EOTe5vHUUsCIiX+Oc\nmwhMBDCzMcDjQFtgpHNuREXX0e0KRcQbiXi8wo+KMLN2wAnOuXGUBeyFZjbPzB4zs5rpPq+AFRFv\nxGIVf1TQHcAvU88XAbc5584GPgDuSvdhjQhExBuZPMllZnUAc87NSR16zjm39cvnwOi09WSsGhER\nv5wNzDrg9QwzOz31vBuwJN0C6mBFxBsZ3gdrlI0CvtQPGG1m+4ANQN90CyhgRcQbmbzZi3Nu2Nde\nLwU6HswaClgR8UYiHq0LDTSDFREJSKAd7Nt/eTzI5QUo3bcv7BK8d9nwa8IuQSpI94MVEQlIxCYE\nClgR8Yc6WBGRgEQsXxWwIuIP3a5QRCQguuG2iEhAItbAKmBFxB9RGxHoQgMRkYCogxURb8QjthFW\nASsi3tA+WBGRgESsgdUMVkQkKOpgRcQbGhGIiAQkaveDVcCKiDei1sFqBisiEhB1sCLijYg1sApY\nEfFH1EYEClgR8UbE8lUBKyL+iNrNXhSwIuKNiOWrAlZE/KEZrIhIQCKWrwpYEfFH1DpYXWggIhIQ\ndbAi4o1M3ovAzG4HLgbygIeBucBEIAm8C/R3zpWWt4Y6WBHxRixW8Ud5zKwLcCbQEegMNANGAoOd\nc2cBMeCSdPUoYEXEG7FYrMKPNL4LrACeA14CpgFtKetiAaYD3dMtohGBiMi/qwc0By4CjgZeBOLO\nuWTq/R1A7XSLKGBFxBsZ3ETwBbDaObcXcGa2m7IxwZdqAlvTLaIRgYh4Ix6PVfiRxpvAeWYWM7PG\nQAEwKzWbBTgfmJ9ukSrfwa5YtZoHx09g/MjfMOje+/liyxYA/rlhIye1PJ77hwwKuUI/vLva8eBj\nTzBu2FBWr3mfX49+hLzcXFocezQ/u/F64nH9WX+olr+7klGjH2bC78ew2v2d+4aPIh6Pk5eXy9C7\nf0G9ukeEXWLgMrUP1jk3zczOBhZR1oj2Bz4ExptZHrAKeDbdOlU6YCf+6RlemTmb/Px8gP1hun3H\nDvreeju3/rhvmOV5Y9IzU3ll1htUz68GwNAHx/Czfn1o3aolD098ilfnzOOCbl3CLTLLPf7EU7z0\nyqvUqF4dgPtH/Jbbf3Yzx1sLpkx9nsefeJKBN/805Cqzi3Nu4Dcc7nwwaxx022Bm1Q72M1HVrHEj\nht89+N+Oj530By6/tCf1q8Cf+JWhaaOGDDvgJ4HPNn1B61YtAWh9QkuWrXwvrNK80axpE377wH37\nXw/79T0cby0AKCkuoVqeN//blitT27Qy5VsD1sx6mtk6M1trZj844K3plVBXpeh2didycr7axG/e\nspVFS5fR87tpd2BIBXXrdCY5icT+100aNmDJ8ncBmL/wHXbt3hNWad7ocU7Xr/xerl+vHgDL/raC\nPz7zLFdf8YNv+6hXMrhNKyPKGxHcCbShLISfMbN859wkyjbYemvmvDc5r1sXEgcEgmTWL275P0aM\nfZRHn55MmxNakZtbpSdVgXn1tZmMmzCJMaOGc8Thh4ddTqWI2K0Iyg3Yvc65LQBmdgkw28w+puwy\nMW8tXLqM66+8POwyvPbmoiXc+/NbqFOrFg88PI4z250adkneeemVV3n2uReYMHYMtWvXCrucSpNN\nX9v9kZmNBIY453aY2WXADKBO5ZQWjnWf/IOmjRuGXYbXvtOkET8e9Avyq+XRtvVJdDq9XdgleaWk\npIT7R4yiUYOGDBh4OwDtTj2F/jdcH3JlVU8smfzmhtTMcoCrgCnOuaLUsQbA7c65ARVZvPAf73vd\n7UZB6b59YZfgvWp164ddQpWQV6vuIbefMweNrXDmdL//xsDb3W/tYJ1zxZTdOebAYxuBCoWriEhl\ny6YZrIhIVoll0QxWRCSrqIMVEQlI1L4yRgErIt6IWL4qYEXEH+pgRUQCErF81f1gRUSCog5WRLwR\ni9h9hRWwIuKNqI0IFLAi4o2oXWgQrX5aRMQj6mBFxBsaEYiIBET7YEVEAlKBr+OuVJrBiogERB2s\niHgjYhMCBayI+EMzWBGRoERs6KmAFRFvqIMVEQlIxPJVASsi/lAHKyISkEznq5kdCSwBegDVgWnA\nmtTbjzjnJpf3eQWsiPgjgwlrZrnA74FdqUNtgZHOuREVXSNi59xERCJjODAW+GfqdVvgQjObZ2aP\nmVnNdAsoYEXEG/FErMKP8phZb+Bz59yMAw4vAm5zzp0NfADcla4ejQhExBsZPMn1v0DSzLoDbYAn\ngIudcxtS7z8HjE63iAJWRLyRqXxNdakAmNkbwI3AC2b2E+fcIqAbZSe/yqWAFRGpmH7AaDPbB2wA\n+qb7gAJWRPwRwD5Y51yXA152PJjPKmBFxBtR+04uBayIeCNqAattWiIiAQm0g1336uIglxeg/olN\nwi7Be/MfmhV2CVXCBSP6H/IaEbsVgUYEIuKPqI0IFLAi4g3dTUtEJCjRylcFrIj4Qx2siEhAFLAi\nIkGJ2MZTBayIeCNqHWzE8l5ExB/qYEXEG9oHKyISEAWsiEhQNIMVEaka1MGKiDci1sAqYEXEH1Hb\npqWAFRFvxBLRmnpGqxoREY+ogxURf0RrQqCAFRF/aAYrIhIQXWggIhKQWDxap5WiVY2IiEfUwYqI\nP6I1IVDAiog/NIMVEQmKdhGIiARD27RERIKSoRGBmSWA8YABSeBGYDcwMfX6XaC/c6603HIyUo2I\nSATEYrEKP9LoCeCc6wgMBoYCI4HBzrmzKDuddkm6RRSwIuKP2EE8yuGcex7om3rZHNgKtAXmpo5N\nB7qnK0cBKyLeyGAHi3Ou2MwmAaOBPwAx51wy9fYOoHa6NRSwIiLfwjl3LdCCsnls9QPeqklZV1su\nBayI+CMeq/ijHGZ2tZndnnpZBJQCi82sS+rY+cD8dOVU+V0EifxqHHPpuayb/gaxRIJGZ55KMpkk\nWVLC+rkLKdm1J+wSs1pxcTH3PTaBTzd9wb59xVx78UW8vmABX2zbDsCGTZs44dhj+OWPbwy50uzW\n8ebvU7x7LwC7Nm/nw/nLOfF7nUmWllL4+TZWTJlddu7bcxm8F8FUYIKZzQNygQHAKmC8meWlnj+b\nbpGqHbCxGI06taO0pASAhmecwqdvLWXP5q3UOf5Y6p3cko0Ll4VcZHab8dYCah12GENu6MP2nTvp\nPeSXTB01DIDthYX83/3D+MkVl4dcZXaL5yQgBgsfeX7/sVN7n8/a1xbz+ep1tL6yB0e2PIrP3vso\nvCIrSab2wTrnCoHvf8NbnQ9mnYMKWDOrDpQ657xo6xq0b8OWVe9Tr01LANbPfpviXbuBsl+oZCp4\n5T/X9fR2dD2tLVDWQCUO+EqPx597ge9170a9OnVCqs4PNRvXI5Gbw2l9exKLx/n7KwvYvv5zcmtU\nAyCnWi6lJeVu15SAlBuwZtYK+DWwhbKzaI8CJWb2U+fctEqoLzC1jzuKkt17KFy/YX/Afhmu1Y+s\nyxEnHMdH02aHWaIXauTnA1C0axeDRz9Mn//uBcCW7dtZvHKVutcMKN27jw/fWMYnC9+joF5t2vXp\nyZrXFtGq11n8V492FO/ay+b314ddZuXIsnsRjAWGAEdRNm9oQdnVDNOBrA7YOi2OBqCgSQPyj6hD\nk87t+fj1NyloVJ96bVrx8Yx5lOz2olEP3cYvNnPHgw/Rq1tXzj2jAwBz3llMjzPak4jY/TuzUeHn\nWynctK3s+aZt7CvaTZsrejDvgT+yc+Nmmnc8kZYXd2Tl1HkhVxq8bLtUNu6cmwvMNbOuzrnPAMys\nOPjSgrXu5Tn7nze/sCufvrmYw5o04PDjj+Wjl+dQumdviNX5Y/O2bdwybCQ3X30F7U5otf/44pWr\nuPbii0KszB9NT29JzUZ1WTl1HtVq1SAnP4/CTdsoTv0e3r2tkMOPahRylZUjat8qmy5gnZk9CvR1\nzvUGMLNBwIagC6tssViMhh1OYV9hEc26dwSg6NPP+HzpypAry25PvPQyO4oKmfjiNCa+WPZDz4hb\nB/DxpxtoXL9+yNX54ZNFq2h9eTc63NQLkrB88mxisRhtrjqXZGkpyZJSVkyZk34hybhYMvntezfM\nLA70dM69cMCxq4CpzrmidIu/9+jkKrAxJFz1T2wSdgnee+eZv4VdQpVwwYj+h/zz/abFb1c4c+q1\nOyPweUK5HWzqTjEvfO3YU4FWJCLyH8q2GayISPZQwIqIBENfGSMiEhR1sCIiAVHAiogEQye5RESC\nErEZbLQuexAR8Yg6WBHxRiwWrZ5RASsi3sjgDbczQgErIv7QDFZEpGpQBysi3tA2LRGRoChgRUSC\nEUskwi7hKzSDFREJiDpYEfGHRgQiIsHQSS4RkaDoSi4RkWDohtsiIkHRiEBEJBiawYqIBCXDM1gz\naw/8xjnXxcxOAaYBa1JvP+Kcm1ze5xWwIuKPDM5gzWwgcDVQmDrUFhjpnBtR4XIyVo2IiF/eBy47\n4HVb4EIzm2dmj5lZzXQLKGBFxBuxWKzCj3Scc38G9h1waBFwm3PubOAD4K50a2hEICLeiMUDvRfB\nc865rV8+B0an+4A6WBHxRyxe8cfBm2Fmp6eedwOWpPuAOlgRkYrpB4w2s33ABqBvug8oYEXEG5m+\nkss59xHQIfV8KdDxYD6vgBURf+hCAxGRYAR8kuugxZLJZNg1iIh4SbsIREQCooAVEQmIAlZEJCAK\nWBGRgChgRUQCooAVEQmIAlZEJCC60OAAB969POxafGRmucDjwFFANeBXzrkXQy3KM2aWAMYDBiSB\nG51z74ZbVdWlDjYldffyR4H8sGvx2FXAF865s4DzgIdCrsdHPQGccx2BwcDQcMup2hSw//L1u5dL\n5j0DDEk9jwHFIdbiJefc8/zrLk/Nga3l/O0SMI0IUpxzfzazo8Kuw2fOuZ0Aqa/aeJayDksyzDlX\nbGaTgF7A98KupypTByuVysyaAXOAJ51zT4ddj6+cc9cCLYDxZlYQdj1VlTpYqTRm1gB4DbjJOTcr\n7Hp8ZGZXA02dc/cBRUBp6iEhUMBKZboDOBwYYmZfzmLPd87tCrEm30wFJpjZPCAXGKD/vuHR7QpF\nRAKiGayISEAUsCIiAVHAiogERAErIhIQBayISEAUsCIiAVHAiogE5P8BYL4sCVSUCnAAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x117363d90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(y_test, lr_cls.predict(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 高斯混合模型\n",
    "\n",
    "#### 建立模型\n",
    "\n",
    "在本案例中，我们使用`Python`语言实现高斯混合模型。我们把实现过程分为两个部分\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ 计算Expectation步\n",
    "\n",
    "在这一步，我们计算每条数据记录$x_{n}$分属K个标签的概率$\\gamma_{n}(z_{k}) = p(z_{k} = 1 \\vert x_{n}, \\theta)$。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''expectation_step函数\n",
    "\n",
    "输入参数：X: 数据集\n",
    "        pis: p(z = 1)的概率\n",
    "        mus: 高斯分布的均值\n",
    "        sigmas: 高斯分布的协方差\n",
    "\n",
    "返回参数： $x_{n}$分属K个标签的概率矩阵\n",
    "\n",
    "'''\n",
    "def expectation_step(X, pis, mus, sigmas):\n",
    "    \n",
    "    import numpy as np\n",
    "    from scipy.stats import multivariate_normal\n",
    "    \n",
    "    gammas  = np.array([value[0]*multivariate_normal(mean = value[1], cov = value[2]).pdf(X) for value in zip(pis, mus, sigmas)])\n",
    "    return gammas / gammas.sum(axis = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ 计算Maximization步\n",
    "\n",
    "在这一步，我们以完全数据的对数极大似然函数为目标，更新参数`pis`, `mus`, `sigmas`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''maximization_step函数\n",
    "\n",
    "输入参数：X: 数据集\n",
    "        gammas：$x_{n}$分属K个标签的概率矩阵\n",
    "        pis: p(z = 1)的概率\n",
    "        mus: 高斯分布的均值\n",
    "        sigmas: 高斯分布的协方差\n",
    "'''\n",
    "\n",
    "def maximization_step(X, gammas, pis, mus, sigmas):\n",
    "    \n",
    "    # 更新pis\n",
    "    pis = gammas.sum(axis = 1)  / gammas.shape[1]\n",
    "\n",
    "    \n",
    "    # 更新mus\n",
    "    mus = np.multiply(1.0/gammas.sum(axis = 1).reshape(-1, 1), np.dot(gammas, X))\n",
    "    \n",
    "    \n",
    "    # 更新sigmas\n",
    "    \n",
    "    for k in range(len(sigmas)):\n",
    "      \n",
    "            \n",
    "        centered_X = X - mus[k]\n",
    "            \n",
    "        sigmas[k] = np.dot(np.multiply(gammas[k], centered_X.T), centered_X) / gammas[k].sum()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "更新最大似然函数的下界`lower_bound`。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''cal_lower_bound函数\n",
    "\n",
    "输入参数：X: 数据集\n",
    "        gammas：$x_{n}$分属K个标签的概率矩阵\n",
    "        pis: p(z = 1)的概率\n",
    "        mus: 训练得到的高斯分布的均值\n",
    "        sigmas: 训练得到的高斯分布的协方差\n",
    "\n",
    "返回参数：对数最大似然函数的下界\n",
    "\n",
    "'''\n",
    "\n",
    "def cal_lower_bound(X, gammas, pis, mus, sigmas):\n",
    "    \n",
    "    from scipy.stats import multivariate_normal\n",
    "\n",
    "    lower_bound = 0\n",
    "        \n",
    "    for gamma, pi, mu, sigma in zip(gammas, pis, mus, sigmas):\n",
    "    \n",
    "        lower_bound  += np.multiply(gamma, np.log(pi*multivariate_normal(mu, sigma).pdf(X)))\n",
    "\n",
    "    return lower_bound.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来，我们实现`train_GMM`函数，计算各个高斯分布的统计特征。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''train_GMM函数\n",
    "\n",
    "输入参数：X: 数据集\n",
    "        k: 高斯分布的个数\n",
    "        max_iter: 迭代的最高次数\n",
    "        alpha: 最大似然函数下界收敛的阈值\n",
    "        \n",
    "        \n",
    "返回参数：(mus, sigmas)以元组形式返回参数\n",
    "        mus: 高斯分布的均值 K times d\n",
    "        sigmas：高斯分布的协方差 d times d\n",
    "        \n",
    "'''\n",
    "def train_GMM(X, K, max_iter, alpha):\n",
    "\n",
    "    indicator = 0\n",
    "    lower_bound = 0\n",
    "   \n",
    "    # 初始化\n",
    "    N, d = X.shape\n",
    "    d = d - 1\n",
    "\n",
    "    # 初始化pi，一维数组，即 K times 1的向量\n",
    "    pis = np.random.random(K)\n",
    "    pis /= pis.sum()\n",
    "\n",
    "    # 初始化mu，二维数组，即 K times d的矩阵\n",
    "    #mus = np.random.random((K, d))\n",
    "    mus = np.array([X[X['label'] == num].mean()[:-1] for num in range(K)])\n",
    "\n",
    "    # 初始化sigmas  三维数组， 即长度为K，元素为 d times d 矩阵的数组\n",
    "    sigmas = np.array([np.eye(d)] * K)\n",
    "    \n",
    "    X_hat = X.iloc[:,:-1].values\n",
    "    while indicator <= max_iter:\n",
    "    \n",
    "        lower_bound_old = lower_bound\n",
    "    \n",
    "        # E-step\n",
    "        gammas = expectation_step(X_hat, pis, mus, sigmas)\n",
    "        \n",
    "        # M-step\n",
    "        maximization_step(X_hat, gammas, pis, mus, sigmas)\n",
    "        \n",
    "        # Update 下界\n",
    "        lower_bound = cal_lower_bound(X_hat, gammas, pis, mus, sigmas)\n",
    "    \n",
    "        # 检查终止条件\n",
    "        if abs(lower_bound - lower_bound_old) < alpha:\n",
    "            print 'reach stopping criterion!'\n",
    "            break\n",
    "    \n",
    "        # 更新迭代次数\n",
    "        indicator += 1\n",
    "    return (mus, sigmas)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "计算高斯分布的参数之后，我们需要判定样本来自于各个分布的概率大小。也就是说将样本代入分布的密度函数，计算其密度，取最大值所在的分布作为该样本的总体分布，并将样本归属于样本所在的类。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "'''predict函数\n",
    "\n",
    "输入参数： X：数据集\n",
    "        mus: 各个高斯分布的均值  K time d\n",
    "        sigmas: 各个高斯分布的协方差   K times 1 元素为d times d 矩阵\n",
    "\n",
    "输出参数：数据集的各个数据记录所对应的标签(如0,1,2)\n",
    "\n",
    "'''\n",
    "\n",
    "def predict(X, mus, sigmas):\n",
    "    \n",
    "    from scipy.stats import multivariate_normal\n",
    "\n",
    "    labels = np.zeros(X.shape[0])\n",
    "    \n",
    "    for index, item in enumerate(X.iloc[:,:-1].values):\n",
    "    \n",
    "        probs = [multivariate_normal(mu, sigma).pdf(item) for mu, sigma in zip(mus, sigmas)]\n",
    "    \n",
    "        labels[index] = probs.index(max(probs))\n",
    "        \n",
    "    return labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 算法验证\n",
    "\n",
    "我们人为生成300个分别来自3个不同高斯分布的数据样本，组成验证数据集。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 生成来自高斯混合分布的数据\n",
    "# 数据集来自3个高斯分布，每个分布产生100个数据样本，数据集的维度为2\n",
    "#np.random.seed(5)\n",
    "\n",
    "mus = [[0, 4], [-2, 0], [4, 4]]\n",
    "\n",
    "sigs = [[[3, 0], [0, 0.5]], [[1, 0], [0, 2]], [[1, 0], [0,1]]]\n",
    "\n",
    "\n",
    "# 数据集X\n",
    "multi_data = [np.random.multivariate_normal(mu, sig, 100) for mu, sig in zip(mus, sigs)]\n",
    "\n",
    "data = np.vstack(multi_data)\n",
    "\n",
    "## 数据集的类别标签为0，1，2\n",
    "labels = [[value]*100 for value in range(3)]\n",
    "## 把嵌套数组labels拉平\n",
    "multi_labels = [y for x in labels for y in x]\n",
    "\n",
    "\n",
    "# 数据集标签合并\n",
    "final_data = []\n",
    "\n",
    "for v1, v2 in zip(data, multi_labels):\n",
    "    list_v1 = v1.tolist()\n",
    "    list_v1.append(v2)\n",
    "    final_data.append(list_v1) \n",
    "\n",
    "##np.random.shuffle(final_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "把数据集转化为`Pandas`中的`DataFrame`格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.230925</td>\n",
       "      <td>4.333235</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.339284</td>\n",
       "      <td>4.856608</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.486634</td>\n",
       "      <td>3.798487</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.479102</td>\n",
       "      <td>3.959988</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.025719</td>\n",
       "      <td>4.627129</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         x1        x2  label\n",
       "0  0.230925  4.333235      0\n",
       "1 -0.339284  4.856608      0\n",
       "2  0.486634  3.798487      0\n",
       "3  0.479102  3.959988      0\n",
       "4  0.025719  4.627129      0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_data = pd.DataFrame(final_data, columns = ['x1', 'x2', 'label'])\n",
    "final_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "测试数据集准备好之后，我们把`final_data`代入函数中进行计算。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reach stopping criterion!\n"
     ]
    }
   ],
   "source": [
    "mus, sigmas = train_GMM(final_data, 3, 1000, 1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.0165078 ,  3.90888693],\n",
       "       [-1.92344288, -0.07807007],\n",
       "       [ 3.9587935 ,  3.98219016]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 根据训练集生成的均值\n",
    "mus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[ 2.79398806,  0.07186567],\n",
       "        [ 0.07186567,  0.45399172]],\n",
       "\n",
       "       [[ 1.2524342 ,  0.11559339],\n",
       "        [ 0.11559339,  2.00399424]],\n",
       "\n",
       "       [[ 0.95237931, -0.09953576],\n",
       "        [-0.09953576,  0.88318031]]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 根据训练集生成的协方差\n",
    "sigmas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.  0.  0.  0.  0.  0.  0.  2.  0.  2.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  2.  2.  2.  0.  0.  0.  2.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  2.  0.  0.  0.  2.  0.  0.  0.  0.  0.  0.\n",
      "  2.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  0.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  0.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.  1.\n",
      "  1.  1.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.\n",
      "  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  0.  2.  2.\n",
      "  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.\n",
      "  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.\n",
      "  2.  2.  2.  2.  2.  2.  2.  2.  0.  2.  2.  2.  2.  2.  2.  2.  2.  2.\n",
      "  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.  2.]\n"
     ]
    }
   ],
   "source": [
    "# 在训练集上预测的标签\n",
    "\n",
    "label_pred = predict(final_data, mus, sigmas)\n",
    "print label_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9533333333333334"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 在测试集上的精度\n",
    "diff = label_pred - final_data['label'].values\n",
    "float(len(diff[diff == 0]))/len(diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.mixture import GaussianMixture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.95"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmm_model = GaussianMixture(n_components = 3, covariance_type = 'diag', init_params='random', max_iter=200)\n",
    "\n",
    "gmm_model.means_init = np.array([final_data[final_data['label'] == num].mean(axis = 0)[:-1] for num in range(3)])\n",
    "\n",
    "gmm_model.fit(final_data.iloc[:,:-1])\n",
    "\n",
    "y_pred = gmm_model.predict(final_data.iloc[:,:-1])\n",
    "\n",
    "\n",
    "\n",
    "diff = y_pred  - final_data['label'].values\n",
    "\n",
    "float(len(diff[diff == 0]))/len(diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.00755769,  3.99859889],\n",
       "       [-2.11044479,  0.06870325],\n",
       "       [ 4.15251596,  4.16263999]])"
      ]
     },
     "execution_count": 424,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmm_model.means_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2.74385235,  0.59128555],\n",
       "       [ 1.04629793,  1.49777959],\n",
       "       [ 1.34015489,  0.99332608]])"
      ]
     },
     "execution_count": 425,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmm_model.covariances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  2.98573219e-02,   3.26413015e+00],\n",
       "       [  2.99997156e+00,   2.07361901e-03],\n",
       "       [  4.26287198e+00,   5.06328795e+00]])"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array([final_data[final_data['label'] == num].mean(axis = 0)[:-1] for num in range(3)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 真实数据集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用`sklearn`中的`GaussianMixture`模型，查看数据集是否适合使用高斯混合模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.10726408689748812"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmm_model = GaussianMixture(n_components = 3, covariance_type = 'diag', init_params='random', max_iter=200)\n",
    "\n",
    "#gmm_model.means_init = np.array([new_data[new_data['label'] == num].mean(axis = 0)[:-1] for num in range(3)])\n",
    "\n",
    "gmm_model.fit(new_data.iloc[:,:-1])\n",
    "\n",
    "y_pred = gmm_model.predict(new_data.iloc[:,:-1])\n",
    "\n",
    "\n",
    "diff = y_pred  - new_data['label'].values\n",
    "\n",
    "float(len(diff[diff == 0]))/len(diff)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "准确率很低，也就是说经过预处理之后的数据集不适合使用高斯混合模型来进行分类，它们的分布可能并不服从高斯分布。可以尝试其他常用的分类模型，比如决策树等。如果使用自定义的模型，去拟合数据集，会得到 `LinAlgError: singular matrix`的错误，也就是说自定义模型并没有考虑含有特殊值的数据集，因此，模型如何改进可以留给读者继续思考。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = pd.read_csv('iris.csv', names=['x1','x2','x3','x4','label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    x1   x2   x3   x4        label\n",
       "0  5.1  3.5  1.4  0.2  Iris-setosa\n",
       "1  4.9  3.0  1.4  0.2  Iris-setosa\n",
       "2  4.7  3.2  1.3  0.2  Iris-setosa\n",
       "3  4.6  3.1  1.5  0.2  Iris-setosa\n",
       "4  5.0  3.6  1.4  0.2  Iris-setosa"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    x1   x2   x3   x4  label\n",
       "0  5.1  3.5  1.4  0.2      0\n",
       "1  4.9  3.0  1.4  0.2      0\n",
       "2  4.7  3.2  1.3  0.2      0\n",
       "3  4.6  3.1  1.5  0.2      0\n",
       "4  5.0  3.6  1.4  0.2      0"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data['label'] = test_data['label'].map({'Iris-setosa':0, 'Iris-versicolor': 1, 'Iris-virginica':2})\n",
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.94"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gmm_model = GaussianMixture(n_components = 3, covariance_type = 'diag', init_params='random', max_iter=200)\n",
    "\n",
    "gmm_model.means_init = np.array([test_data[test_data['label'] == num].mean(axis = 0)[:-1] for num in range(3)])\n",
    "\n",
    "gmm_model.fit(test_data.iloc[:,:-1])\n",
    "\n",
    "y_pred = gmm_model.predict(test_data.iloc[:,:-1])\n",
    "\n",
    "\n",
    "diff = y_pred  - test_data['label'].values\n",
    "\n",
    "float(len(diff[diff == 0]))/len(diff)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 2, 2, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 2, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2])"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reach stopping criterion!\n"
     ]
    }
   ],
   "source": [
    "mus, sigmas = train_GMM(test_data, 3, 1000, 1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "        0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  1.,  1.,\n",
       "        1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "        1.,  1.,  1.,  2.,  1.,  2.,  1.,  2.,  1.,  1.,  1.,  1.,  1.,\n",
       "        1.,  1.,  1.,  1.,  1.,  2.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,\n",
       "        1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  1.,  2.,  2.,  2.,  2.,\n",
       "        2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,\n",
       "        2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,\n",
       "        2.,  2.,  2.,  1.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,  2.,\n",
       "        2.,  2.,  2.,  2.,  2.,  2.,  2.])"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_pred = predict(test_data, mus, sigmas)\n",
    "label_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.9667\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      1.00      1.00        50\n",
      "          1       0.98      0.92      0.95        50\n",
      "          2       0.92      0.98      0.95        50\n",
      "\n",
      "avg / total       0.97      0.97      0.97       150\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVgAAAD7CAYAAAAvk4y0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEolJREFUeJzt3XmYVNWZx/FvVbOIbG4xmSRGE6LHGAX33YABHTRjjNkz\nIQsYNQ4mkjiuLI6KIYIgo45jglEZjTNOXLKQIWFcAKNowKWFQI4x7pmsRhRphfR0zR9dMAS1q1vq\n9r196vt5nnqeulXW8bXU3/Py3nNvlSqVCpKk+ivnXYAkpcqAlaSMGLCSlBEDVpIyYsBKUkYMWEnK\nSK+8C5CkIgohPAS8VD18ErgYuB6oACuA8THGto7WMGAlaTMhhK2AUoxxxCav/RCYFGNcGEK4Gjge\nuL2jdQxYSXqtYcDWIYQFtOfkecB+wKLq+/OBo8kzYIfuPNzLxDK2bPlteZcg1UWfQduXtnSNrmTO\no08v6ujv1wJcClwD7Ep7oJZijBvWXwMMrvX3sIOVpNd6DHi8GqiPhRCep72D3WAgsLrWIu4ikJSM\nUqnU6UcN44CZACGEtwODgAUhhBHV948B7qm1iB2spGSUSnXrGb8DXB9C+BntuwbGAX8C5oQQ+gCr\ngFtqLWLASkpGmS0e4wIQY1wP/P3rvDW8K+sYsJKS0Yk/+ncrA1ZSMsr1GxHUhQErKRlF62CLFfeS\nlBA7WEnJaCo15V3CXzFgJSWjaCMCA1ZSMsoFC1hnsJKUETtYSckoFaxnNGAlJaOpbMBKUiZKdbpU\ntl6KFfeSlBA7WEnJ8FJZScqI+2AlKSNF2wdrwEpKRtFOchmwkpLhDFaSMuIMVpIy4gxWkjLiDFaS\nMuKIQJIy4ohAkjLiiECSMlK0bVrFqkaSEmIHKykZnuSSpIw0FWxE0PABe/OP5/DymrUA/ObZ3zHn\nyhu4aOY5UIHH45NcPPkyKpVKzlWmoa2tjamXXEr81a/o07sPF0w6l3ft9M68y0pKo3/HRdtFUKy4\n72Z9+vaBUokTPz2BEz89gSlnfpMzJ4/nyku/wxc/8RUowZFHH553mcm4a+Fi1q1bz3evncOE005l\nxuzL8y4pOX7HxdLQARveN4R+W/Xl6hsu5Zp/v4yh++zB+/bajWX3PwLAzxY+wMGH75dzlel4qLmZ\nww89CIBhe+3JylW/zLmi9DT6d1wqlTr96A4NPSJ49ZV1zP32zdz6H/PY+d3v5Kq50//qi29Z28KA\ngf1zrDAta9e2MKD/gI3H5XITra2t9OrV0P8Z1lWjf8dFGxE0xrf+Bp568lmeeeo5AJ5+8jlWv/AS\ne+y128b3t+6/NWteejmv8pLTv//WrG1p2XjcVmlrmP/xu0ujf8c96kKDEMLdQN/NXi4BlRjjoZlV\n1U1O+OSx7Lr7e7h40mW8ZcftGTBga5bcs4z9D96bZfc/wuEjDmLpkofzLjMZ+wwbysLF9zL6qJE0\nL1/BrkOG5F1Schr9O+5pHew5wBzgBKA1+3K61203/5ipM8/l+luugApMOesSVv/5Rc7/5pn07tOb\nJx5/mv/+r0V5l5mMkSOGs+SBpYwZdzIVKlw0ZWLeJSWn0b/jou2DLdXaghRCOBN4PMZ4e1cXH7rz\ncPc3ZWzZ8tvyLkGqiz6Dtt/idBx36D90OnOuve+qzNO45nAmxjgj6yIkqR561AxWknqSes9gQwg7\nAg8CR9E+Jr0eqAArgPExxrYO66lrNZKUo3rugw0h9Aa+BbxSfWkWMCnGeATtJ/uPr7WGASspGeVS\nqdOPTrgUuBr4n+rxfsCGs97zgVE163kz/xCSVET16mBDCF8E/hhj/Ommy8cYN5xEWwMMrlWPM1hJ\nyajjSa5xQCWEMArYG/g3YMdN3h8IrK61iB2sJG0mxviBGOPwGOMI4BHg88D8EMKI6l9yDHBPrXXs\nYCUlo5ztLq0zgDkhhD7AKuCWWh8wYCUlo6lc/z+UV7vYDYZ35bMGrKRkFO1SWWewkpQRO1hJySh7\nqawkZaNoIwIDVlIyetr9YCWpxyhYvhqwktJhBytJGfF+sJKUEU9ySVJGHBFIUkYKlq8GrKR02MFK\nUkY8ySVJGbGDlaSMFCxfvZuWJGXFDlZSMrK44faWMGAlJaNoIwIDVlIyinaSq1j9tCQlxA5WUjLc\nBytJGfFmL5KUkaZysQLWGawkZSTTDnbZ8tuyXF7AsYeckncJybt93tS8S2gIfQZtv8VrOCKQpIwU\nbEJgwEpKhx2sJGWkYPlqwEpKR9Gu5DJgJSXDCw0kKSMFa2ANWEnpKNqIwAsNJCkjdrCSklEu2EZY\nA1ZSMtwHK0kZqVcDG0JoAuYAAagAXwZeBa6vHq8AxscY2zqspz7lSFJSjgOIMR4GTAIuBmYBk2KM\nRwAl4PhaixiwkpJRKpU6/ehIjPH7wMnVw52B1cB+wKLqa/OBUbXqcUQgKRn1vB9sjLE1hDAXOAH4\nOHBUjLFSfXsNMLjWGnawkpJRrw52gxjjF4DdaJ/H9tvkrYG0d7UdMmAlaTMhhM+FEM6tHrYAbcCy\nEMKI6mvHAPfUWscRgaRk1HGX1m3AdSGExUBvYAKwCpgTQuhTfX5LrUUMWEnJqNc+2BjjWuCTr/PW\n8K6sY8BKSkbBrjMwYCWlo2g3ezFgJSWjYPlqwEpKh/cikKSMFCxfDVhJ6ShaB+uFBpKUETtYScmo\n570I6sGAlZSMgk0IDFhJ6XAGK0kNwg5WUjIK1sAasJLSUbRflXVEALS1tXHhtOl8dtxJjD1lPM88\n+1zeJSVnm+0GcdMdV7PTu9/ONtsN4sIrzmbW3AuZfeNU/mant+ZdXnL+vHo1Hxozjqca7L/let9w\ne0vZwQJ3LVzMunXr+e61c2hevoIZsy/nipnT8y4rGU29mphw/imsX7cegJPO+Bx3zlvMop8uYdiB\n7+dd734Hv3329zlXmY7W1la+cflV9O3bN+9SGl6XO9gQQnL/1h5qbubwQw8CYNhee7Jy1S9zrigt\np/zj55n3nwt4/g8vAPD+fXZnh7dtz/RrpjDyQ0fQvPQXOVeYltlzruNjx45mh+22y7uUblcqdf7R\nHd4wYEMIx4UQng4hPB5C+NQmb83vhrq61dq1LQzoP2DjcbncRGtra44VpePoj4zgxRdeYtm9zRtf\ne9vb38LLL67lrC9dyB9+9yc+deJHcqwwLT9acCfbDB7MIfvvm3cpuehJI4KJwN60h/D3QghbxRjn\n0v574Enp339r1ra0bDxuq7TRq5fTk3oYfcIHqVQq7HvwUIbsvgtnT/sKbW1tLLl7KQD33/0gY0//\nTM5VpuOHC+6gVCrx84ebeeyJJ5kyYzaz/mkiO2y3bd6ldYuetItgfYzxBYAQwvHAXSGEZ4BKB5/p\nkfYZNpSFi+9l9FEjaV6+gl2HDMm7pGR8/QtTNj6fed0FzL7wW4z96mc48AP7csePFrPX/u/j6cef\nzbHCtMy5dNrG5yefOZHzvnpqw4Qr9KxLZZ8KIcwCJscY14QQPgr8FNime0rrPiNHDGfJA0sZM+5k\nKlS4aMrEvEtK2tXT53LGhady3Kf+lrUvt/CNs2bnXZKUiY4CdhwwhmrHGmN8NoRwJHBuB5/pkcrl\nMlPOPSvvMpJ3xtjzNz4/+6SLcqykMXx7xsV5l9Dtinap7BsGbIyxFbh+s9d+T/vP10pS4RQsX90H\nKykdpR40g5WkHsUOVpIy0mNmsJLU0xQsXw1YSemwg5WkjBQsX71doSRlxQ5WUjJK5WL1jAaspGQU\nbURgwEpKRtEuNChWPy1JCbGDlZQMRwSSlBH3wUpSRor2s90GrCRtJoTQG7gW2AXoC0wFVtJ+C9cK\nsAIYH2Ns62gdT3JJSkYdf1V2DPB8jPEIYDRwJTALmFR9rQQcX2sRA1ZSMur4q7LfAyZvWBZoBfYD\nFlVfmw+MqrWIIwJJ6ahTyxhjfBkghDAQuAWYBFwaY9zwo69rgMHdVI4k5a+OHSwhhJ2Au4EbYow3\nAZvOWwcCq2utYcBKSka9ZrAhhLcCC4CzY4zXVl9+OIQwovr8GOCeWvU4IpCUjDrugz0P2BaYHELY\nMIs9Hbg8hNAHWEX76KBDBqykZNQrX2OMp9MeqJsb3pV1DFhJ6SjYlVzOYCUpI3awkpJRbipWB2vA\nSkqGN3uRpIwULF+dwUpSVuxgJaWjYC2sASspGUX7TS4DVlIyihawzmAlKSN2sD3cvMVX5l1C8g47\nYEzeJTSEB5+4Y4vXKNgI1oCVlI6ijQgMWEnJ8EIDScpKsfLVgJWUDjtYScqIAStJWSnYxlMDVlIy\nitbBFizvJSkddrCSkuE+WEnKiAErSVlxBitJjcEOVlIyCtbAGrCS0lG0bVoGrKRklJqKNfUsVjWS\nlBA7WEnpKNaEwICVlA5nsJKUES80kKSMlMrFOq1UrGokKSF2sJLSUawJgQErKR3OYCUpK+4ikKRs\n1HubVgjhIOCSGOOIEMJ7geuBCrACGB9jbOvo857kkpSOcqnzjxpCCGcB1wBbVV+aBUyKMR5B+7T3\n+JrlbME/iiQVSqlU6vSjE34NfHST4/2ARdXn84FRtRYwYCWlo9SFRw0xxluBv2y6eoyxUn2+Bhhc\naw1nsJKSkfGlspvOWwcCq2t9wA5Wkjrn4RDCiOrzY4B7an3AgAXa2tq4cNp0PjvuJMaeMp5nnn0u\n75KS9egvVnLi+NPzLiNJ226/DT/+2U3s8p6d2P3972Xu7Vdyzc2Xceb5pxXuJiiZqeNJrtdxBnBB\nCGEJ0Ae4pdYHHBEAdy1czLp16/nutXNoXr6CGbMv54qZ0/MuKznX3XgT836ygH79+uVdSnJ69Wpi\n4tQJrFu3HoCJ3/gaMy74Fx59aCWnfn0soz/8Qeb/4M6cq8xeve9FEGN8Cji4+vwxYHhXPt+lakII\n/UIIfbvymZ7goeZmDj/0IACG7bUnK1f9MueK0rTTO97BrGlT8y4jSRPOPYVbb5rHH3//PAA7vu0t\nPPrQSgCaH1zB3gfsmWd53abOuwi2WIcBG0LYI4Tw/RDCdSGEUcAqYGUI4e+6pbpusnZtCwP6D9h4\nXC430drammNFaRp15HB69WrKu4zkHPexo3nhzy+y5J5lG1/7zTO/Zd8DhwLwgZGH0K/fVm/0cWWo\n1ojgamAysAvt84bdgFdp3wM2L9PKulH//luztqVl43FbpY1evZyeqGf48CdGU6lUOPCwfQl7DOGC\nmWfzz9O+zdhTP8NJXx3Dw0tXsH79X2ovlIIedi+CcoxxEbAohHBkjPEPACGEpNq7fYYNZeHiexl9\n1Eial69g1yFD8i5J6rSTPv31jc+/ddNMpk2azWFHHsikr03jxdUvceb5p3Hfop/nWGH3KdrJvFoB\nG0MI1wAnxxi/CBBCOAf4XdaFdaeRI4az5IGljBl3MhUqXDRlYt4lSVvkmSd/w7/eOJ1XX1nHsvsf\n4d6FDRKwBftV2VKlUnnDN0MIZeC4GOMPNnltDHBbjLHlDT9Ytf6l5994cdVF218a5I9+OTrsgDF5\nl9AQHnziji1uP/+09L5OZ84OBxyaebvbYQdbvVPMDzZ77cZMK5KkN6uHjQgkqcfoaTNYSeo5DFhJ\nyoY/GSNJWbGDlaSMGLCSlA1PcklSVgo2gy3WZQ+SlBA7WEnJKJWK1TMasJKSUe8bbm8pA1ZSOpzB\nSlJjsIOVlAy3aUlSVgxYScpGqalYv/nmDFaSMmIHKykdjggkKRue5JKkrHgllyRlwxtuS1JWHBFI\nUjacwUpSVpzBSlJGCjaDLVbcS1JC7GAlJcMZrCRlpFQu1r0IDFhJ6SjYSa5iVSNJCbGDlZSMel3J\nFUIoA1cBw4B1wJdijI93dR07WEnpKJU6/+jYR4CtYoyHAOcAM99MOQaspGSUyk2dftRwOPATgBjj\n/cD+b6aeTEcEfQZtX6w9E9Kb8OATd+RdgjqpjpkzCHhxk+P/DSH0ijG2dmURO1hJeq2XgIGbHJe7\nGq5gwErS67kXOBYghHAwsPzNLOIuAkl6rduBo0II9wElYOybWaRUqVTqWpUkqZ0jAknKiAErSRlx\nBruJEMJBwCUxxhF515KiEEJv4FpgF6AvMDXG+MNci0pMCKEJmAMEoAJ8Oca4It+qGpcdbFUI4Szg\nGmCrvGtJ2Bjg+RjjEcBo4Mqc60nRcQAxxsOAScDF+ZbT2AzY//dr4KN5F5G47wGTq89LQJf3Fapj\nMcbvAydXD3cGVudYTsNzRFAVY7w1hLBL3nWkLMb4MkAIYSBwC+0dluosxtgaQpgLnAB8PO96Gpkd\nrLpVCGEn4G7ghhjjTXnXk6oY4xeA3YA5IYT+edfTqOxg1W1CCG8FFgCnxRjvzLueFIUQPge8M8Y4\nDWgB2qoP5cCAVXc6D9gWmBxC2DCLPSbG+EqONaXmNuC6EMJioDcwwe83P17JJUkZcQYrSRkxYCUp\nIwasJGXEgJWkjBiwkpQRA1aSMmLASlJGDFhJysj/AdJGJ/JVjPHpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10e7460d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate(label_pred, test_data['label'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_data = pd.read_table('seed.txt',sep='\\t', names=['x1','x2','x3','x4','x5','x6','x7','label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_data['label'] = seed_data['label'] - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x1</th>\n",
       "      <th>x2</th>\n",
       "      <th>x3</th>\n",
       "      <th>x4</th>\n",
       "      <th>x5</th>\n",
       "      <th>x6</th>\n",
       "      <th>x7</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15.26</td>\n",
       "      <td>14.84</td>\n",
       "      <td>0.8710</td>\n",
       "      <td>5.763</td>\n",
       "      <td>3.312</td>\n",
       "      <td>2.221</td>\n",
       "      <td>5.220</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14.88</td>\n",
       "      <td>14.57</td>\n",
       "      <td>0.8811</td>\n",
       "      <td>5.554</td>\n",
       "      <td>3.333</td>\n",
       "      <td>1.018</td>\n",
       "      <td>4.956</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14.29</td>\n",
       "      <td>14.09</td>\n",
       "      <td>0.9050</td>\n",
       "      <td>5.291</td>\n",
       "      <td>3.337</td>\n",
       "      <td>2.699</td>\n",
       "      <td>4.825</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13.84</td>\n",
       "      <td>13.94</td>\n",
       "      <td>0.8955</td>\n",
       "      <td>5.324</td>\n",
       "      <td>3.379</td>\n",
       "      <td>2.259</td>\n",
       "      <td>4.805</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16.14</td>\n",
       "      <td>14.99</td>\n",
       "      <td>0.9034</td>\n",
       "      <td>5.658</td>\n",
       "      <td>3.562</td>\n",
       "      <td>1.355</td>\n",
       "      <td>5.175</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      x1     x2      x3     x4     x5     x6     x7  label\n",
       "0  15.26  14.84  0.8710  5.763  3.312  2.221  5.220      0\n",
       "1  14.88  14.57  0.8811  5.554  3.333  1.018  4.956      0\n",
       "2  14.29  14.09  0.9050  5.291  3.337  2.699  4.825      0\n",
       "3  13.84  13.94  0.8955  5.324  3.379  2.259  4.805      0\n",
       "4  16.14  14.99  0.9034  5.658  3.562  1.355  5.175      0"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seed_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gmm_model = GaussianMixture(n_components = 3, covariance_type = 'diag', init_params='random', max_iter=200)\n",
    "\n",
    "gmm_model.means_init = np.array([seed_data[seed_data['label'] == num].mean(axis = 0)[:-1] for num in range(3)])\n",
    "\n",
    "gmm_model.fit(seed_data.iloc[:,:-1])\n",
    "\n",
    "y_pred = gmm_model.predict(seed_data.iloc[:,:-1])\n",
    "\n",
    "\n",
    "diff = y_pred  - seed_data['label'].values\n",
    "\n",
    "float(len(diff[diff == 0]))/len(diff)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
